# -*- coding: utf-8
'''
Created on 13.02.2010

@author: Admin
'''
from mechanize._mechanize import LinkNotFoundError
import re, urllib
from mechanize import Browser
from BeautifulSoup import BeautifulSoup
from htmlentitydefs import name2codepoint

def _html_unescape(str):
    def entity_replacer(m):
        entity = m.group(1)
        if entity in name2codepoint:
            return unichr(name2codepoint[entity])
        else:
            return m.group(0)

    def ascii_replacer(m):
        cp = int(m.group(1))
        if cp <= 255:
            return unichr(cp)
        else:
            return m.group(0)
    s =    re.sub(r'&#(\d+);',  ascii_replacer, str, re.U)
    return re.sub(r'&([^;]+);', entity_replacer, s, re.U)
#root_url = 'http://search.yahoo.com//'
#
#br = Browser()
#br.set_handle_robots(False)
#start_page = br.open(root_url)
#br.select_form(nr = 0)
#br.set_value("test", id = "yschsp")
#page = br.submit()


class BingBrowser():
    countries = {
                    'algeria': 'DZ',
                    'argentina': 'AR',
                    'australia': 'AU',
                    'austria': 'AT',
                    'belgium': 'BE',
                    'brazil': 'BR',
                    'canada': 'CA',
                    'chile': 'CL',
                    'colombia': 'CO',
                    'costa rica': 'CR',
                    'czech republic': 'CZ',
                    'denmark': 'DK',
                    'dominican republic': 'DO',
                    'egypt': 'EG',
                    'el salvador': 'SV',
                    'estonia': 'EE',
                    'finland': 'FI',
                    'france': 'FR',
                    'germany': 'DE',
                    'greece': 'GR',
                    'guatemala': 'GT',
                    'honduras': 'HN',
                    'hungary': 'HU',
                    'iceland': 'IS',
                    'india': 'IN',
                    'indonesia': 'ID',
                    'ireland': 'IE',
                    'italy': 'IT',
                    'japan': 'JP',
                    'kenya': 'KE',
                    'korea': 'KR',
                    'luxembourg': 'LU',
                    'malaysia': 'MY',
                    'mexico': 'MX',
                    'morocco': 'MA',
                    'netherlands': 'NL',
                    'new Zealand': 'NZ',
                    'nicaragua': 'NI',
                    'norway': 'NO',
                    'panama': 'PA',
                    'peru': 'PE',
                    'poland': 'PL',
                    'portugal': 'PT',
                    'romania': 'RO',
                    'russia': 'RU',
                    'singapore': 'SG',
                    'south africa': 'ZA',
                    'spain': 'ES',
                    'sweden': 'SE',
                    'switzerland': 'CH',
                    'taiwan': 'TW',
                    'thailand': 'TH',
                    'tunisia': 'TN',
                    'turkey': 'TR',
                    'united kingdom': 'GB',
                    'united states': 'US',
                    'vietnam': 'VN',
                 }
    languages = {
                    'albanian': 'sq',
                    'arabic': 'ar',
                    'bulgarian': 'bg',
                    'catalan': 'ca',
                    'chinese simplified': 'zh_chs',
                    'chinese traditional': 'zh_cht',
                    'croatian': 'hr',
                    'czech': 'cs',
                    'danish': 'da',
                    'dutch': 'nl',
                    'english': 'en',
                    'estonian': 'et',
                    'finnish': 'fi',
                    'french': 'fr',
                    'german': 'de',
                    'greek': 'el',
                    'hebrew': 'he',
                    'hungarian': 'hu',
                    'icelandic': 'is',
                    'indonesian': 'id',
                    'italian': 'it',
                    'japanese': 'ja',
                    'korean': 'ko',
                    'latvian': 'lv',
                    'lithuanian': 'lt',
                    'malay': 'ms',
                    'norwegian': 'nb',
                    'persian': 'fa',
                    'polish': 'pl',
                    'portuguese': 'pt_pt',
                    'romanian': 'ro',
                    'russian': 'ru',
                    'serbian': 'sr',
                    'slovak': 'sk',
                    'slovenian': 'sl',
                    'spanish': 'es',
                    'swedish': 'sv',
                    'thai': 'th',
                    'turkish': 'tr',
                 }

    def __init__(self, query, number_of_results, language = None, country = None):
        self.mech = Browser()
        self.mech.set_handle_robots(False)
        self.query = query
        self.num = number_of_results
        if language: language = self.languages[language.lower()]
        self.language = language
        if country: country = self.countries[country.lower()]
        self.country = country
        self.collected_res = 0
        self.make_url()
        self.res_pages = []


    def make_url(self):
        if self.language:
            self.query += " language:%s" %self.language
        if self.country:
            self.query += " loc:%s" %self.country

        self.url = "http://www.bing.com/search?q=%s" %urllib.quote_plus(self.query)

    def get_page(self):
        if self.collected_res == 0:
            page = self.mech.open(self.url)
#            self.res_pages.append(page.read())
#            self.collected_res += 9
        else:
            first = 1 + 10*len(self.res_pages)
#            link = self.mech.find_link(text = str(len(self.res_pages) + 1), url_regex = re.compile("/search\?q=.*?sc=1-33&amp;first=%s&amp;" %first))
            link = self.mech.find_link(text = str(len(self.res_pages) + 1), url_regex = re.compile(".*?first=%s.*" %first))
            page = self.mech.follow_link(link)
        self.res_pages.append(page.read())
        self.collected_res += 10

    def get_results(self):
        while True:
            try:
                self.get_page()
            except LinkNotFoundError:
                break
            if self.collected_res >= self.num:
                break
        return self.res_pages

class YahooBrowser():
    countries = {
                    'algeria': 'dz',
                    'argentina': 'ar',
                    'australia': 'au',
                    'austria': 'at',
                    'belgium': 'be',
                    'brazil': 'br',
                    'canada': 'ca',
                    'chile': 'cl',
                    'colombia': 'co',
                    'costa rica': 'cr',
                    'czech republic': 'cz',
                    'denmark': 'dk',
                    'dominican republic': 'do',
                    'egypt': 'eg',
                    'el salvador': 'sv',
                    'estonia': 'ee',
                    'finland': 'fi',
                    'france': 'fr',
                    'germany': 'de',
                    'greece': 'gr',
                    'guatemala': 'gt',
                    'honduras': 'hn',
                    'hungary': 'hu',
                    'iceland': 'is',
                    'india': 'in',
                    'indonesia': 'id',
                    'ireland': 'ie',
                    'italy': 'it',
                    'japan': 'jp',
                    'kenya': 'ke',
                    'korea': 'kr',
                    'luxembourg': 'lu',
                    'malaysia': 'my',
                    'mexico': 'mx',
                    'morocco': 'ma',
                    'netherlands': 'nl',
                    'new zealand': 'nz',
                    'nicaragua': 'ni',
                    'norway': 'no',
                    'panama': 'pa',
                    'peru': 'pe',
                    'poland': 'pl',
                    'portugal': 'pt',
                    'romania': 'ro',
                    'russia': 'ru',
                    'singapore': 'sg',
                    'south africa': 'za',
                    'spain': 'es',
                    'sweden': 'se',
                    'switzerland': 'ch',
                    'taiwan': 'tw',
                    'thailand': 'th',
                    'tunisia': 'tn',
                    'turkey': 'tr',
                    'united kingdom': 'uk',
                    'united states': 'us',
                    'vietnam': 'vn',
                        }
    languages = {
                    'albanian': 'lang_sq',
                    'arabic': 'lang_ar',
                    'bulgarian': 'lang_bg',
                    'catalan': 'lang_ca',
                    'chinese simplified': 'lang_zh-CN',
                    'chinese traditional': 'lang_zh-TW',
                    'croatian': 'lang_hr',
                    'czech': 'lang_cs',
                    'danish': 'lang_da',
                    'dutch': 'lang_nl',
                    'english': 'lang_en',
                    'estonian': 'lang_et',
                    'finnish': 'lang_fi',
                    'french': 'lang_fr',
                    'german': 'lang_de',
                    'greek': 'lang_el',
                    'hebrew': 'lang_iw',
                    'hungarian': 'lang_hu',
                    'icelandic': 'lang_is',
                    'indonesian': 'lang_id',
                    'italian': 'lang_it',
                    'japanese': 'lang_ja',
                    'korean': 'lang_ko',
                    'latvian': 'lang_lv',
                    'lithuanian': 'lang_lt',
                    'malay': 'lang_ms',
                    'norwegian': 'lang_no',
                    'persian': 'lang_fa',
                    'polish': 'lang_pl',
                    'portuguese': 'lang_pt',
                    'romanian': 'lang_ro',
                    'russian': 'lang_ru',
                    'serbian': 'lang_sr',
                    'slovak': 'lang_sk',
                    'slovenian': 'lang_sl',
                    'spanish': 'lang_es',
                    'swedish': 'lang_sv',
                    'thai': 'lang_th',
                    'turkish': 'lang_tr',
                 }

    def __init__(self, query, number_of_results, language = None, country = None):
        self.mech = Browser()
        self.mech.set_handle_robots(False)
        self.mech.addheaders.append(("Accept-Charset", "utf-8"))
        self.query = query
        self.num = number_of_results
        self.res_per_page = min(100, self.num + (10 - self.num%10))
        if language: language = self.languages[language.lower()]
        self.language = language
        if country: country = self.countries[country.lower()]
        self.country = country
        self.collected_res = 0
        self.res_pages = []
        self.make_url()


    def make_url(self):
        self.url = "http://search.yahoo.com/search?n=%s&ei=UTF-8&p=%s&fl=0&va_vt=any&vo_vt=any&ve_vt=any&vp_vt=any&vd=all&vst=0&vf=all&vm=p&" %(self.res_per_page, urllib.quote_plus(self.query))
#        va_vt=any&vo_vt=any&ve_vt=any&vp_vt=any&vd=all&vst=0&vf=all&vm=p&fl=0
        if self.language:
            self.url += "&vl=%s&fl=1" %self.language
        if self.country:
            self.url += "&vc=%s" %self.country

    def get_page(self):
        if self.collected_res == 0:
            page = self.mech.open(self.url)
#            self.res_pages.append(page.read())
#            self.collected_res += 9
        else:
            link = self.mech.find_link(text = str(len(self.res_pages) + 1))
            page = self.mech.follow_link(link)
        self.res_pages.append(page.read())
        self.collected_res += self.res_per_page

    def get_results(self):
        while True:
            try:
                self.get_page()
            except LinkNotFoundError:
                break
            if self.collected_res >= self.num:
                break
        return self.res_pages

class YahooSearch():
    def __init__(self, html):
        self.soup = BeautifulSoup(html)
    def get_search_results(self):
        return self.soup.findAll('div', "res")

    def extract_info(self, item):
        ref = item.find("a")
    #    print ref
        url = ref["href"]
        title = ' '.join([r.strip() for r in ref.findAll(text = True)])

        title = _html_unescape(title)
        desc = item.find("div", "abstr")
        desc = _html_unescape(' '.join([r.strip() for r in desc.findAll(text = True)]))
        return {'url': url, 'title': title, 'desc': desc}

class GoogleSearch():
    def __init__(self, html):
        self.soup = BeautifulSoup(html)
    def get_search_results(self):
        return self.soup.findAll('li', "g w0")

    def extract_info(self, item):
        ref = item.find("a")
    #    print ref
        url = ref["href"]
        title = ''.join(ref.findAll(text = True))

        title = _html_unescape(title)
        desc = item.find("div", "s")
        for div in desc.findAll("div"):
            div.extract()
        for div in desc.findAll("cite"):
            div.extract()
        for div in desc.findAll("span"):
            div.extract()
#        desc = _html_unescape(''.join([r.strip() for r in desc.findAll(text = True)]))
        desc = _html_unescape(''.join(desc.findAll(text = True)))
        return {'url': url, 'title': title, 'desc': desc}

class BingSearch():
    def __init__(self, html):
        self.soup = BeautifulSoup(html)
    def get_search_results(self):
#        res = self.soup.find("div", id = "results")
#        assert len(res.findAll('li')) == 10, len(res.findAll('li'))
#        return self.soup.findAll('li')
        return self.soup.find("ul", id = "wg0").contents

    def extract_info(self, item):
        ref = item.find("a")
    #    print ref
        url = ref["href"]
        title = ''.join(ref.findAll(text = True))
        title = _html_unescape(title)
        desc = item.find("p")
        if desc:
            desc = _html_unescape(''.join(desc.findAll(text = True)))
        else:
            desc = ""
        return {'url': url, 'title': title, 'desc': desc}
#soup = BeautifulSoup(page.read())

def main(query, num_of_results_per_search_engine, language = None, country = None):
    res = {}
    y = YahooBrowser(query, num_of_results_per_search_engine, language, country)
    y.get_results()
    j = 0
    for page in y.res_pages:
        print >> open(r'F:\dev\python\searches\test_content\yahoo_crawl.html', "w"), page
        s = YahooSearch(page)
        for item in s.get_search_results():
            j += 1
            info = s.extract_info(item)
            res[info['url']] = info
#            print info['url']
#            print info['title']
#            print info['desc']
#            print
            if j >= num_of_results_per_search_engine:
                break
    b = BingBrowser(query, num_of_results_per_search_engine, language, country)
    b.get_results()
    j = 0
    for page in b.res_pages:
        print >> open(r'F:\dev\python\searches\test_content\yahoo_crawl.html', "w"), page
        s = BingSearch(page)
        for item in s.get_search_results():
            j += 1
            info = s.extract_info(item)
            res[info['url']] = info
            if j >= num_of_results_per_search_engine:
                break
    print "total %s results:" %len(res)
    for info in res.values():
        print info['url']
        print info['title']
        print info['desc']
        print

main("python", 10, "english")